import json

table_list = []
def read_table(path):
    with open(path, 'r') as file:
        # 逐行读取 JSONL 文件
        for line in file:
            # 将每一行的 JSON 字符串解析为字典
            data = json.loads(line)

            # 提取 "header" 内容
            header = data.get('header', [])
            # 打印结果
            table_list.append(header)
read_table('C:\\Users\\86130\\Downloads\\wiki\\dev.tables.jsonl')
read_table('C:\\Users\\86130\\Downloads\\wiki\\test.tables.jsonl')
read_table('C:\\Users\\86130\\Downloads\\wiki\\train.tables.jsonl')

similarity = 0
db = len(table_list)
for i in range(db):
    for j in range(i + 1, db):
        des1 = table_list[i]
        des2 = table_list[j]
        len1 = len(des1)
        len2 = len(des2)
        intersection = len(set(des1) & set(des2))
        sim = intersection / (len1 + len2 -intersection)
        similarity = similarity + sim
print(2 * similarity / (db * (db - 1)))
